#!/usr/bin/python
# -*- coding: utf-8 -*-
"""中文分词"""

import re
import os


class SEG(object):
    """简单分词类
    按给定字典精确分词，未匹配的按字母切分自动分词
    """
    
    def __init__(self, dict_path='dic'):
        self._dicts = {}
        self._init_dicts(dict_path)
        
    def _load_dicts(self, dict_path):
        if os.path.isdir(dict_path):
            paths = []
            for root, subs, files in os.walk(dict_path):
                for filepath in files:
                    if filepath.endswith('.dic'):
                        paths.append(os.path.join(root, filepath))
        else:
            paths = [dict_path]
        for p in paths:
            with open(p, 'rb') as f:
                for line in f:
                    line = line.strip().decode('utf-8')
                    if line: # 2个字以上
                        word = line.split(None, 1)[0]
                        if len(word) > 1:
                            # 过滤“x的”关键词，例如指的
                            if len(word) == 2 and word.endswith(u'的'):
                                continue
                            yield word

    def _init_dicts(self, dict_path):
        for word in self._load_dicts(dict_path):
            word = unichr(11) + word
            last_p, last_c = None, None
            p = self._dicts
            for c in reversed(word):
                c = c.lower()
                if p is None:
                    last_p[last_c] = {} # 上一级创建节点
                    p = last_p[last_c]
                if c not in p:
                    p[c] = None
                    last_p = p
                    last_c = c
                p = p[c] # next
    
    def _binary_seg(self, s):
        """二分切割"""
        length = len(s)
        if length != 1:
            return (s[i-2:i] for i in xrange(length, 1, -1))
        return [] # 单字不做分词
    
#    REPLACE_BLANK_RE = re.compile(u"。|，|,|！|…|!|《|》|<|>|\"|\'|:|：|？|\?|、|\||“|”|‘|’|；|—|（|）|·|\(|\)|\s|和|的|了", re.IGNORECASE)
    REPLACE_BLANK_RE = re.compile(u"""
        &nbsp;
        |
        [
        。，,！…!《》<>\"\':：？、“”‘’；—（）·%#@~=
        ﹫﹢﹦﹤﹟﹩﹠﹪﹡﹨【】〖〗
        \(\)\?\s\|\-\{\}\*\&\^\$\/
        和的了
        ]
        +""", re.IGNORECASE | re.VERBOSE) 
    SPLIT_RE = re.compile(ur'([0-9a-z\-\+#@_\.]+)', re.I)
    def _process_unrecognised(self, s):
        words = []
        if s:
            pieces = self.REPLACE_BLANK_RE.split(s)
            for c in reversed(pieces):
                mc = self.SPLIT_RE.split(c) # 使用特殊符号和因为分词
                if not mc:
                    continue
                for r in reversed(mc):
                    if not r:
                        continue
                    if self.SPLIT_RE.search(r):
                        words.append(r)
                    else:
#                        if 1 < len(r) < 5:
#                            words.append(r)
                        words.extend(self._binary_seg(r))
        return words
    
    REMOVE_HTML_TAG_RE = re.compile(ur'<(style|script)[^<>]*>.*?</\1>|</?[a-z][a-z0-9]*[^<>]*>|<!--.*?-->', re.I)
    def cut(self, text, remove_html_tag=False):
        """分词
 |  a  |  b  |  c  |  d  |  e  |  f  |
                                     iz
 |  a  |  b  |  c  |  d  |  e  |  f  |
                               i     z
 |  a  |  b  |  c  |  d  |  e  |  f  |
                  i-j          i     z    j = 2 (found: de, f)
        """
        if isinstance(text, str):
            text = text.decode('utf-8', 'ignore')
        if remove_html_tag:
            text = self.REMOVE_HTML_TAG_RE.sub(u'', text)
        p, end = self._dicts, len(text) # dict pointer, end pos
        i, j, z = end, 0, end # 成功匹配字符的结束位置, 成功匹配字符数, 未匹配字符的结束位置
        recognised = []
        mem = None
        while i - j > 0:
            word = text[i-j-1].lower()
            if word not in p: # not found
                p = self._dicts
                if mem is not None:
                    i, j, z = mem
                    if i < z:
                        recognised.extend(self._process_unrecognised(text[i:z])) # 将未匹配的字符特殊处理
                    start = i - j
                    recognised.append(text[start:i])
                    i, j, z = start, 0, start
                    mem = None
                else:
                    i, j = i - 1, 0
            else: # word in dictionary
                p, j = p[word], j + 1 # next
                if chr(11) in p: # 关键词匹配成功
#                    if j <= 2 and len(p) > 1: # 尽量匹配更多字符
                    if len(p) > 1: # 尽量匹配更多字符
                        mem = i, j, z
                    else:
                        p = self._dicts
                        if i < z:
                            recognised.extend(self._process_unrecognised(text[i:z]))
                        start = i - j
                        recognised.append(text[start:i])
                        i, j, z = start, 0, start
                        mem = None
        recognised.extend(self._process_unrecognised(text[i-j:z]))
        return recognised